<?php
namespace App\Noxinfluencer\Controllers;
require_once '../../../vendor/autoload.php';

use Medoo\Medoo;
use phpspider\core\phpspider;
use phpspider\core\requests;
use phpspider\core\selector;

/* Do NOT delete this comment */
/* 不要删除这段注释 */

$db = new Medoo([
    'database_type' => 'mysql',
    'database_name' => 'youtube',
    'server' => '127.0.0.1',
    'username' => 'root',
    'password' => 'wewrjjdsl@oo888rmds'
]);

$all_country = ['us','ca','fi','se','no','is','dk','lt','by','ru','ua','pl','cz','hu','de','at','ch','uk','ie','nl','be','lu','fr','mc','ro','rs','it','es','pt','au','nz','tr','bh','ir','iq','il','kw','ae','ar','br','co','mx','jp','kr','my','sg','vn','th'];



$all_country = $db->select("y_country",['country_simple_name','country_china_name'],['country_simple_name'=>$all_country]);


//$not_search_country = ['us','in','jp','kr','th','id','uk','tw','vn','br','ad'];
$not_search_country = [];


$pageSize = 300;


$configs = array(
    'name' => 'noxinfluencer',
    'log_show' => false,
    'log_file' => "/data/spider/noxinfluencer_spider_test.log",
//    'save_running_state' => true,
    'tasknum' => 1,
    'max_depth' => 1,
    //数据库配置
    'db_config' => array(
        'host'  => '127.0.0.1',
        'port'  => 3306,
        'user'  => 'root',
        'pass'  => 'wewrjjdsl@oo888rmds',
        'name'  => 'youtube',
    ),
    'queue_config' => array(
        'host' => '127.0.0.1',
        'port' => 6379,
        'pass' => '',
        'db' => 1,
        'prefix' => 'phpspider',
        'timeout' => 30,
    ),
    /*'export' => array(
        'type' => 'csv',
        'file' => './data/Youtube.csv', // data目录下
    ),*/
    'export' => array(
        'type' => 'db',
        'table' => 'user_nox',  // 如果数据表没有数据新增请检查表结构和字段名是否匹配
    ),
//爬取的域名列表
    'domains' => array(
        'cn.noxinfluencer.com'
    ),
//抓取的起点
    'scan_urls' => array(
        "https://cn.noxinfluencer.com/youtube/search/?isvideo=0&category=all&follower=0&area=us&exposure=0&published=0&language=all&noxScoreLevel=0&pageNum=1",
    ),
//列表页实例
    'list_url_regexes' => array(
        "https://cn.noxinfluencer.com/youtube/search/?isvideo=0&category=all&follower=0&area=us&exposure=0&published=0&language=all&noxScoreLevel=0&pageNum=1",
    ),
//内容页实例
    'content_url_regexes' => array(
        "https://cn.noxinfluencer.com/youtube/channel/[A-Za-z0-9]+",
    ),
    'max_try' => 5,

    'client_ip' => array(
        '192.168.0.2','192.168.0.3','192.168.0.4','192.168.0.5','192.168.0.6','192.168.0.7','192.168.0.8','192.168.0.9',
        '192.168.1.2','192.168.1.3','192.168.1.4','192.168.1.5','192.168.1.6','192.168.1.7','192.168.1.8','192.168.1.9',
        '192.168.2.2','192.168.2.3','192.168.2.4','192.168.2.5','192.168.2.6','192.168.2.7','192.168.2.8','192.168.2.9',
        '192.168.3.2','192.168.3.3','192.168.3.4','192.168.3.5','192.168.3.6','192.168.3.7','192.168.3.8','192.168.3.9',
        '192.168.4.2','192.168.4.3','192.168.4.4','192.168.4.5','192.168.4.6','192.168.4.7','192.168.4.8','192.168.4.9',
        '192.168.5.2','192.168.5.3','192.168.5.4','192.168.5.5','192.168.5.6','192.168.5.7','192.168.5.8','192.168.5.9',
    ),

    'fields' => array(
        array(
            'name' => "name",
            'selector' => "//div[@class='title']/span",
            'required' => true,
        ),
        array(
            'name' => "channel_id",
            'selector' => "//div[@class='info-header']/div[@class='socials']/a[@class='icon-wrapper']/@href",
            'required' => true,
        ),
        array(
            'name' => "fans_num",
            'selector' => "//div[@class='info-bottom']//div[@class='column']//div[@class='sub-title']//span[@class='strong']",
            'required' => false,
        ),
        array(
            'name' => "video_look_num",
            'selector' => "//div[@class='info-bottom']//span[@class='strong']",
            'required' => false,
            'repeated' => true,
        ),
        array(
            'name' => "video_avg_look_num",
            'selector' => "//div[@class='info-bottom']//span[@class='strong']",
            'required' => false,
            'repeated' => true,
        ),
        array(
            'name' => "video_num",
            'selector' => "//div[@class='info-bottom']//div[@class='column'][last()]//span[@class='strong']",
            'required' => false,
        ),
        array(
            'name' => "country",
            'selector' => "//div[@class='info-content']//span[@class='subtitle underline']",
            'required' => false,
        ),
        array(
            'name' => "language",
            'selector' => "//div[@class='info-content']//span[@class='subtitle'][last()]",
            'required' => false,
        ),
        array(
            'name' => "head_img",
            'selector' => "//div[@class='avatar-wrapper']/img/@src",
            'required' => false,
        ),
        array(
            'name' => "category",
            'selector' => "//div[@class='info-container']//div[@class='info-header']//div[@class='tags']//a",
            'required' => false,
        ),
        array(
            'name' => "register_time",
            'selector' => "//div[@class='info-content']/span[@class='title']/span[@class='subtitle']",
            'required' => false,
        ),

    ),
);

$spider = new phpspider($configs);



$spider->on_extract_field = function($fieldname, $data, $page)
{
    if ($fieldname == 'channel_id')
    {
        if (strpos($data, "channel/") !== false)
        {
            $channel = explode('channel/',$data);
            return $channel[1];
        }
    }

    if ($fieldname == 'name')
    {
        return str_replace('&amp;','&',$data);
    }

    if ($fieldname == 'fans_num')
    {
        return str_replace(['万','亿','.'],['0000','00000000',''],$data);
    }

    if ($fieldname == 'video_look_num')
    {
        return str_replace(['万','亿','.'],['0000','00000000',''],$data[1]);
    }

    if ($fieldname == 'video_avg_look_num')
    {
        return str_replace(['万','亿','.'],['0000','00000000',''],$data[2]);
    }

    if ($fieldname == 'video_num')
    {
        return str_replace(['万','亿','.'],['0000','00000000',''],$data);
    }

    if ($fieldname == 'country')
    {
        return preg_replace("/(\s|\ \;| |\xc2\xa0)/","",$data);
    }

    if ($fieldname == 'language')
    {
        return preg_replace("/(\s|\ \;| |\xc2\xa0)/","",$data);
    }

    if ($fieldname == 'category')
    {
        return preg_replace("/(\s|\ \;| |\xc2\xa0)/","",$data);
    }

    return $data;
};


$spider->on_fetch_url = function($url, $spider)
{
    if (strpos($url, "mailto") !== false)
    {
        return false;
    }
    return $url;
};


$spider->on_start = function ($spider) use ($all_country,$not_search_country,$pageSize)
{
    foreach ($all_country as $key => $value){
        $country_simple_name = strtolower($value['country_simple_name']);
        if(!in_array($country_simple_name,$not_search_country)){
            // 生成列表页URL入队列
            for ($i = 1; $i <= $pageSize; $i++)
            {
                $url = "https://cn.noxinfluencer.com/youtube/search/?isvideo=0&category=all&follower=0&area={$country_simple_name}&exposure=0&published=0&language=all&noxScoreLevel=0&pageNum={$i}";
                $spider->add_scan_url($url);
            }
        }
    }
};


$spider->start();






